from scrapy import Request, Spider
import json
import time
import datetime


def beautify_str(result):
    return result.replace('\n', '').replace('\r', '').replace('\xa0', '').replace("'", '’').strip()


def get_ts_list():
    result = []
    flag = True
    st_str = "2019-04-22 08:00:00"

    while flag:
        print(st_str)
        st_datetime = datetime.datetime.strptime(st_str, "%Y-%m-%d %H:%M:%S")
        now_datetime = datetime.datetime.now()
        if st_datetime < now_datetime:
            ts = int(time.mktime(time.strptime(st_str, "%Y-%m-%d %H:%M:%S")) * 1000)
            result.append(ts)
            st_str = (st_datetime + datetime.timedelta(hours=3)).strftime("%Y-%m-%d %H:%M:%S")
        else:
            flag = False
    print(len(result))
    return result


class MaoyanSpider(Spider):
    name = 'maoyan'

    def start_requests(self):
        # ts_list = get_ts_list()
        movieId = 248172

        # for ts in ts_list:
        #     yield Request(
        #         url="http://m.maoyan.com/review/v2/comments.json?movieId={}&userId=-1&offset=0&limit=15&ts={}&type=3".format(movieId, ts),
        #         callback=self.parse,
        #         meta={'movieId': movieId, 'ts': ts}
        #     )

        ts = int(time.time() * 1000)
        yield Request(
            url="http://m.maoyan.com/review/v2/comments.json?movieId={}&userId=-1&offset=0&limit=15&ts={}&type=3".
            format(movieId, ts),
            callback=self.parse,
            meta={
                'movieId': movieId,
                'ts': ts
            }
        )

    def parse(self, response):
        print(response.url)
        movieId = response.meta['movieId']
        ts = response.meta['ts']
        json_data = json.loads(response.text)
        comments = json_data['data']['comments']
        hotComments = json_data['data']['hotComments']
        all_comment = comments + hotComments
        for item in all_comment:
            postdate = int(item['time']) / 1000
            postdatetime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(postdate))
            postdate = postdatetime[:10].replace('-', '')
            tags = []
            tagList = item['tagList']
            for tag in tagList:
                tags.append(tag['name'])
            data = {
                'cid': item['id'],
                'movieId': item['movieId'],
                'content': beautify_str(item['content']),
                'score': item['score'],
                'nick': item['nick'],
                'gender': item['gender'],
                'userId': item['userId'],
                'userLevel': item['userLevel'],
                'upCount': item['upCount'],
                'postdate': postdate,
                'postdatetime': postdatetime,
                'tags': '|'.join(tags),
                'replyCount': item['replyCount']
            }
            # print(data)
            yield data
            if data['replyCount'] > 0:
                yield Request(
                    url='http://m.maoyan.com/mmdb/replies/comment/{}.json?_v_=yes&offset=0'.format(data['cid']),
                    callback=self.parse_more,
                    meta={
                        'movieId': movieId,
                        'cid': data['cid'],
                        'offset': 0
                    }
                )
        offset = json_data['paging']['offset']
        next_page = offset + 15
        hasMore = json_data['paging']['hasMore']
        if hasMore:
            yield Request(
                url="http://m.maoyan.com/review/v2/comments.json?movieId={}&userId=-1&offset={}&limit=15&ts={}&type=3".
                format(movieId, next_page, ts),
                callback=self.parse,
                meta={
                    'movieId': movieId,
                    'ts': ts
                }
            )

    def parse_more(self, response):
        print(response.url)
        cid = response.meta['cid']
        movieId = response.meta['movieId']
        offset = response.meta['offset']
        json_data = json.loads(response.text)
        comments = json_data['cmts']
        for item in comments:
            data = {
                'cid': item['id'],
                'movieId': movieId,
                'content': beautify_str(item['content']),
                'score': '',
                'nick': item['nickName'],
                'gender': '',
                'userId': item['userId'],
                'userLevel': item['userLevel'],
                'upCount': '',
                'postdate': item['time'][:10].replace('-', ''),
                'postdatetime': item['time'] + ':00',
                'tags': '',
                'replyCount': ''
            }
            yield data
        if len(comments):
            offset = offset + 10
            yield Request(
                url='http://m.maoyan.com/mmdb/replies/comment/{}.json?_v_=yes&offset={}'.format(cid, offset),
                callback=self.parse_more,
                meta={
                    'movieId': movieId,
                    'cid': cid,
                    'offset': offset
                }
            )
